{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset: 6000\n",
      "Descriptions: train=6000, test=1000\n",
      "Photos: train=6000, test=1000\n"
     ]
    }
   ],
   "source": [
    "from CapGenerator import load_data as ld\n",
    "\n",
    "data = ld.prepare_dataset('train')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_features, train_descriptions = data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-4-782ec568ef4b>, line 11)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-4-782ec568ef4b>\"\u001b[0;36m, line \u001b[0;32m11\u001b[0m\n\u001b[0;31m    for key Pin descriptions.keys():\u001b[0m\n\u001b[0m              ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils import to_categorical\n",
    "import numpy as np\n",
    "from CapGenerator import generate_model as gen\n",
    "\n",
    "\n",
    "# convert a dictionary of clean descriptions to a list of descriptions\n",
    "def to_lines(descriptions):\n",
    "    all_desc = list()\n",
    "    for key Pin descriptions.keys():\n",
    "        [all_desc.append(d) for d in descriptions[key]]\n",
    "        return all_desc\n",
    "\n",
    "    # fit a tokenizer given caption descriptions\n",
    "    def create_tokenizer(descriptions):\n",
    "        lines = to_lines(descriptions)\n",
    "        tokenizer = Tokenizer()\n",
    "        tokenizer.fit_on_texts(lines)\n",
    "        return tokenizer\n",
    "\n",
    "    # calculate the length of the description with the most words\n",
    "    def max_length(descriptions):\n",
    "        lines = to_lines(descriptions)\n",
    "        return max(len(d.split()) for d in lines)\n",
    "\n",
    "    # create sequences of images, input sequences and output words for an image\n",
    "    def create_sequences(tokenizer, max_length, desc_list, photo):\n",
    "        X1, X2, y = list(), list(), list()\n",
    "        # walk through each description for the image\n",
    "        for desc in desc_list:\n",
    "            # encode the sequence\n",
    "            seq = tokenizer.texts_to_sequences([desc])[0]\n",
    "            # split one sequence into multiple X,y pairs\n",
    "            for i in range(1, len(seq)):\n",
    "             s\n",
    "            # split into input and output pair\n",
    "                in_seq, out_seq = seq[:i], seq[i]\n",
    "                # pad input sequence\n",
    "                in_seq = pad_sequences([in_seq], maxlen=max_length)[0]\n",
    "                # encode output sequence\n",
    "                out_seq = to_categorical([out_seq], num_classes=vocab_size)[0]\n",
    "                # store\n",
    "                X1.append(photo)\n",
    "                X2.append(in_seq)\n",
    "                y.append(out_seq)\n",
    "                return np.array(X1), np.array(X2), np.array(y)\n",
    "\n",
    "            # data generator, intended to be used in a call to model.fit_generator()\n",
    "            def data_generator(descriptions, photos, tokenizer, max_length):\n",
    "                # loop for ever over images\n",
    "                while 1:\n",
    "                    for key, desc_list in descriptions.items():\n",
    "                        # retrieve the photo feature\n",
    "                        photo = photos[key][0]\n",
    "                        print('Photo:')\n",
    "                        print(photo.shape)\n",
    "                        in_img, in_seq, out_word = create_sequences(tokenizer, max_length, desc_list, photo)\n",
    "                        yield [[in_img, in_seq], out_word]\n",
    "\n",
    "                        # prepare tokenizer\n",
    "                        tokenizer = create_tokenizer(train_descriptions)\n",
    "                        vocab_size = len(tokenizer.word_index) + 1\n",
    "                        print('Vocabulary Size: %d' % vocab_size)\n",
    "\n",
    "                        # determine the maximum sequence length\n",
    "                        max_length = max_length(train_descriptions)\n",
    "                        print('Description Length: %d' % max_length)\n",
    "\n",
    "                        # test the data generator\n",
    "                        generator = gen.data_generator(train_descriptions, train_features, tokenizer, max_length)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs, outputs = next(generator)\n",
    "print(inputs[0].shape)\n",
    "print(inputs[1].shape)\n",
    "print(outputs.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.utils import to_categorical\n",
    "from keras.utils import plot_model\n",
    "from keras.models import Model, Sequential\n",
    "from keras.layers import Input\n",
    "from keras.layers import Dense\n",
    "from keras.layers import LSTM\n",
    "from keras.layers import Embedding\n",
    "from keras.layers import Dropout\n",
    "from keras.layers import RepeatVector\n",
    "from keras.layers import TimeDistributed\n",
    "from keras.layers import concatenate\n",
    "from keras.layers import Reshape\n",
    "from keras.layers import merge\n",
    "from keras.layers import GRU\n",
    "from keras.layers import BatchNormalization\n",
    "\n",
    "from IPython.display import SVG\n",
    "from keras.utils.vis_utils import model_to_dot\n",
    "\n",
    "from keras.optimizers import Adam\n",
    "\n",
    "\n",
    "EMBEDDING_DIM = 128\n",
    "lstm_layers = 3\n",
    "dropout_rate = 0.22\n",
    "learning_rate = 0.001\n",
    "\n",
    "# define the captioning model\n",
    "def define_model(vocab_size, max_length):\n",
    "    # feature extractor (encoder)\n",
    "    inputs1 = Input(shape=(4096, ))\n",
    "    fe1 = Dropout(0.5)(inputs1)\n",
    "    fe2 = Dense(EMBEDDING_DIM, activation='relu')(fe1)\n",
    "    fe3 = RepeatVector(max_length)(fe2)\n",
    "\n",
    "    # embedding\n",
    "    inputs2 = Input(shape=(max_length, ))\n",
    "    emb2 = Embedding(vocab_size, 256, mask_zero=True)(inputs2)\n",
    "\n",
    "    # merge inputs\n",
    "    merged = concatenate([fe3, emb2])\n",
    "    # language model (decoder)\n",
    "    input_ = merged\n",
    "    for _ in range(lstm_layers):\n",
    "        input_ = BatchNormalization()(input_)\n",
    "        lstm_out = LSTM(\n",
    "            300,\n",
    "            return_sequences=True,\n",
    "            dropout=dropout_rate,\n",
    "            recurrent_dropout=dropout_rate)(input_)\n",
    "        input_ = lstm_out\n",
    "    outputs = Dense(vocab_size, activation='softmax')(lstm_out)\n",
    "\n",
    "    # tie it together [image, seq] [word]\n",
    "    model = Model(inputs=[inputs1, inputs2], outputs=outputs)\n",
    "    model.compile(\n",
    "        loss='categorical_crossentropy',\n",
    "        optimizer=Adam(lr=learning_rate),\n",
    "        metrics=['accuracy'])\n",
    "    print(model.summary())\n",
    "    plot_model(model, show_shapes=True, to_file='model.png')\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = define_model(200, 34)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display(SVG(model_to_dot(model, show_shapes=True, show_layer_names=True).create(prog='dot', format='svg')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def image_caption_model(vocab_size=2187, embedding_matrix=None, lang_dim=100,\n",
    "            max_caplen=53, img_dim=2048, clipnorm=1):\n",
    "    print('generating vocab_history model v5')\n",
    "    # text: current word\n",
    "    lang_input = Input(shape=(1,))\n",
    "    img_input = Input(shape=(img_dim,))\n",
    "    seq_input = Input(shape=(max_caplen,))\n",
    "    vhist_input = Input(shape=(vocab_size,))\n",
    "\n",
    "    if embedding_matrix is not None:\n",
    "        x = Embedding(output_dim=lang_dim, input_dim=vocab_size, init='glorot_uniform', input_length=1, weights=[embedding_matrix])(lang_input)\n",
    "    else:\n",
    "        x = Embedding(output_dim=lang_dim, input_dim=vocab_size, init='glorot_uniform', input_length=1)(lang_input)\n",
    "\n",
    "    lang_embed = Reshape((lang_dim,))(x)\n",
    "    lang_embed = merge([lang_embed, seq_input], mode='concat', concat_axis=-1)\n",
    "    lang_embed = Dense(lang_dim)(lang_embed)\n",
    "    lang_embed = Dropout(0.25)(lang_embed)\n",
    "\n",
    "    merge_layer = merge([img_input, lang_embed, vhist_input], mode='concat', concat_axis=-1)\n",
    "    merge_layer = Reshape((1, lang_dim+img_dim+vocab_size))(merge_layer)\n",
    "\n",
    "    gru_1 = GRU(img_dim)(merge_layer)\n",
    "    gru_1 = Dropout(0.25)(gru_1)\n",
    "    gru_1 = Dense(img_dim)(gru_1)\n",
    "    gru_1 = BatchNormalization()(gru_1)\n",
    "    gru_1 = Activation('softmax')(gru_1)\n",
    "\n",
    "    attention_1 = merge([img_input, gru_1], mode='mul', concat_axis=-1)\n",
    "    attention_1 = merge([attention_1, lang_embed, vhist_input], mode='concat', concat_axis=-1)\n",
    "    attention_1 = Reshape((1, lang_dim + img_dim + vocab_size))(attention_1)\n",
    "    gru_2 = GRU(1024)(attention_1)\n",
    "    gru_2 = Dropout(0.25)(gru_2)\n",
    "    gru_2 = Dense(vocab_size)(gru_2)\n",
    "    gru_2 = BatchNormalization()(gru_2)\n",
    "    out = Activation('softmax')(gru_2)\n",
    "    \n",
    "    model = Model(input=[img_input, lang_input, seq_input, vhist_input], output=out)\n",
    "    model.compile(loss='categorical_crossentropy', optimizer=RMSprop(lr=0.0001, clipnorm=1.))\n",
    "    plot_model(model, show_shapes=True, to_file='plot.png')\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Activation\n",
    "from keras.optimizers import RMSprop\n",
    "\n",
    "model = image_caption_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SVG(model_to_dot(model, show_shapes=True, show_layer_names=True).create(prog='dot', format='svg'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Model\n",
    "from keras.layers import Input, Dropout, TimeDistributed, Masking, Dense, Lambda, Permute\n",
    "from keras.layers import BatchNormalization, Embedding, Activation, Reshape, Multiply\n",
    "from keras.layers.merge import Add, Concatenate, Average\n",
    "from keras.layers.recurrent import LSTM, GRU, SimpleRNN\n",
    "from keras.regularizers import l2\n",
    "from keras import backend as K\n",
    "\n",
    "def NIC(max_token_length, vocabulary_size, rnn='lstm' ,num_image_features=2048,\n",
    "        hidden_size=512, embedding_size=512, regularizer=1e-8, batch_size= 20):\n",
    "\n",
    "    # word embedding\n",
    "    text_input = Input(shape=(max_token_length, vocabulary_size), name='text')#batch_shape=batch_size,\n",
    "    text_mask = Masking(mask_value=0.0, name='text_mask')(text_input)\n",
    "    text_to_embedding = TimeDistributed(Dense(units=embedding_size,\n",
    "                                        kernel_regularizer=l2(regularizer),\n",
    "                                        name='text_embedding'))(text_mask)\n",
    "\n",
    "    text_dropout = Dropout(.5, name='text_dropout')(text_to_embedding)\n",
    "\n",
    "    # image embedding\n",
    "    image_input = Input(shape=(max_token_length, 14, 14, 512), #batch_shape=batch_size,\n",
    "                                                        name='image')\n",
    "    image_input2 = Reshape((max_token_length, 196, 512), name='reshape1')(image_input)\n",
    "  #  denselastlayer\n",
    "    image_embedding = TimeDistributed(Dense(units=embedding_size,\n",
    "                                      kernel_regularizer=l2(regularizer),\n",
    "                                      name='image_embedding'))(image_input2)\n",
    "    image_dropout = Dropout(.5, name='image_dropout')(image_embedding)\n",
    "\n",
    "    # language model\n",
    "    #recurrent_inputs = [text_dropout, text_dropout]\n",
    "   # merged_input = Add()(recurrent_inputs)\n",
    "    if rnn == 'lstm':\n",
    "        for i in range(max_token_length):\n",
    "            if i == 0:\n",
    "              #  first_input = AVE(units=1,\n",
    "               ##                  name='initial_zero')(image_dropout[:, i, :, :])\n",
    "                getlistlayer = Lambda(lambda x: [x[:, j, :] for j in range(196)])\n",
    "                getim0layer = Lambda(lambda x: x[:, 0, :, :])\n",
    "                temp_layer = getim0layer(image_dropout)\n",
    "                #order_input = getlistlayer(temp_layer)\n",
    "                #order_input = [image_dropout[:, i, j, :] for j in range(196)]\n",
    "                avelayer = Lambda(lambda x:K.mean(x, axis=1))\n",
    "                first_input = avelayer(temp_layer)\n",
    "                #first_input = Average()(order_input)\n",
    "                #first_input = Dense(units=100,\n",
    "                 #                   kernel_regularizer=l2(regularizer),\n",
    "                  #                  name='initial_zero_embed')(first_input)\n",
    "                gettx0layer = Lambda(lambda x: x[:, 0, :])\n",
    "                first_input2 = gettx0layer(text_dropout)\n",
    "                recurrent_inputs = [first_input2, first_input]\n",
    "                merged_input_temp = Add()(recurrent_inputs)\n",
    "                merged_input = Reshape((1, embedding_size), name='reshape2')(merged_input_temp)\n",
    "            else:\n",
    "                getim1layer = Lambda(lambda x: x[:, 1, :, :])\n",
    "                getim2layer = Lambda(lambda x: x[:, 2, :, :])\n",
    "                getim3layer = Lambda(lambda x: x[:, 3, :, :])\n",
    "                getim4layer = Lambda(lambda x: x[:, 4, :, :])\n",
    "                getim5layer = Lambda(lambda x: x[:, 5, :, :])\n",
    "                getim6layer = Lambda(lambda x: x[:, 6, :, :])\n",
    "                getim7layer = Lambda(lambda x: x[:, 7, :, :])\n",
    "                getim8layer = Lambda(lambda x: x[:, 8, :, :])\n",
    "                getim9layer = Lambda(lambda x: x[:, 9, :, :])\n",
    "                getim10layer = Lambda(lambda x: x[:, 10, :, :])\n",
    "                getim11layer = Lambda(lambda x: x[:, 11, :, :])\n",
    "                getim12layer = Lambda(lambda x: x[:, 12, :, :])\n",
    "                getim13layer = Lambda(lambda x: x[:, 13, :, :])\n",
    "                getim14layer = Lambda(lambda x: x[:, 14, :, :])\n",
    "                getim15layer = Lambda(lambda x: x[:, 15, :, :])\n",
    "                getim16layer = Lambda(lambda x: x[:, 16, :, :])\n",
    "                getim17layer = Lambda(lambda x: x[:, 17, :, :])\n",
    "                getim18layer = Lambda(lambda x: x[:, 18, :, :])\n",
    "                getim19layer = Lambda(lambda x: x[:, 19, :, :])\n",
    "                getim20layer = Lambda(lambda x: x[:, 20, :, :])\n",
    "                getim21layer = Lambda(lambda x: x[:, 21, :, :])\n",
    "                getim22layer = Lambda(lambda x: x[:, 22, :, :])\n",
    "                getim23layer = Lambda(lambda x: x[:, 23, :, :])\n",
    "                getim24layer = Lambda(lambda x: x[:, 24, :, :])\n",
    "                getim25layer = Lambda(lambda x: x[:, 25, :, :])\n",
    "                getim26layer = Lambda(lambda x: x[:, 26, :, :])\n",
    "                getim27layer = Lambda(lambda x: x[:, 27, :, :])\n",
    "                getim28layer = Lambda(lambda x: x[:, 28, :, :])\n",
    "                getim29layer = Lambda(lambda x: x[:, 29, :, :])\n",
    "                getim30layer = Lambda(lambda x: x[:, 30, :, :])\n",
    "                getim31layer = Lambda(lambda x: x[:, 31, :, :])\n",
    "                if i == 1:\n",
    "                    outputimsplit = getim1layer(image_dropout)\n",
    "                elif i == 2:\n",
    "                    outputimsplit = getim2layer(image_dropout)\n",
    "                elif i==3:\n",
    "                    outputimsplit = getim3layer(image_dropout)\n",
    "                elif i == 4:\n",
    "                    outputimsplit = getim4layer(image_dropout)\n",
    "                elif i==5:\n",
    "                    outputimsplit = getim5layer(image_dropout)\n",
    "                elif i == 6:\n",
    "                    outputimsplit = getim6layer(image_dropout)\n",
    "                elif i==7:\n",
    "                    outputimsplit = getim7layer(image_dropout)\n",
    "                elif i == 8:\n",
    "                    outputimsplit = getim8layer(image_dropout)\n",
    "                elif i==9:\n",
    "                    outputimsplit = getim9layer(image_dropout)\n",
    "                elif i == 10:\n",
    "                    outputimsplit = getim10layer(image_dropout)\n",
    "                elif i==11:\n",
    "                    outputimsplit = getim11layer(image_dropout)\n",
    "                elif i == 12:\n",
    "                    outputimsplit = getim12layer(image_dropout)\n",
    "                elif i==13:\n",
    "                    outputimsplit = getim13layer(image_dropout)\n",
    "                elif i == 14:\n",
    "                    outputimsplit = getim14layer(image_dropout)\n",
    "                elif i==15:\n",
    "                    outputimsplit = getim15layer(image_dropout)\n",
    "                elif i == 16:\n",
    "                    outputimsplit = getim16layer(image_dropout)\n",
    "                elif i==17:\n",
    "                    outputimsplit = getim17layer(image_dropout)\n",
    "                elif i == 18:\n",
    "                    outputimsplit = getim18layer(image_dropout)\n",
    "                elif i==19:\n",
    "                    outputimsplit = getim19layer(image_dropout)\n",
    "                elif i == 20:\n",
    "                    outputimsplit = getim20layer(image_dropout)\n",
    "                elif i==21:\n",
    "                    outputimsplit = getim21layer(image_dropout)\n",
    "                elif i == 22:\n",
    "                    outputimsplit = getim22layer(image_dropout)\n",
    "                elif i==23:\n",
    "                    outputimsplit = getim23layer(image_dropout)\n",
    "                elif i == 24:\n",
    "                    outputimsplit = getim24layer(image_dropout)\n",
    "                elif i==25:\n",
    "                    outputimsplit = getim25layer(image_dropout)\n",
    "                elif i == 26:\n",
    "                    outputimsplit = getim26layer(image_dropout)\n",
    "                elif i==27:\n",
    "                    outputimsplit = getim27layer(image_dropout)\n",
    "                elif i == 28:\n",
    "                    outputimsplit = getim28layer(image_dropout)\n",
    "                elif i==29:\n",
    "                    outputimsplit = getim29layer(image_dropout)\n",
    "                elif i == 30:\n",
    "                    outputimsplit = getim30layer(image_dropout)\n",
    "                else:\n",
    "                    outputimsplit = getim31layer(image_dropout)\n",
    "                per_out = Permute((2, 1))(outputimsplit)\n",
    "                per_out1 = Dense(units=1,\n",
    "                                   kernel_regularizer=l2(regularizer))(per_out)\n",
    "                dim_change = Dense(units=128,\n",
    "                                   kernel_regularizer=l2(regularizer))(lstm_out)\n",
    "                dim_change2 = Permute((2, 1))(dim_change)\n",
    "                #per_out2 = Reshape((196, 1))(dim_change)\n",
    "                attendout3 = Multiply()([per_out1, dim_change2])\n",
    "                pre_merge = Reshape((1, embedding_size))(attendout3)\n",
    "                gettx1layer = Lambda(lambda x: x[:, 1, :])\n",
    "                gettx2layer = Lambda(lambda x: x[:, 2, :])\n",
    "                gettx3layer = Lambda(lambda x: x[:, 3, :])\n",
    "                gettx4layer = Lambda(lambda x: x[:, 4, :])\n",
    "                gettx5layer = Lambda(lambda x: x[:, 5, :])\n",
    "                gettx6layer = Lambda(lambda x: x[:, 6, :])\n",
    "                gettx7layer = Lambda(lambda x: x[:, 7, :])\n",
    "                gettx8layer = Lambda(lambda x: x[:, 8, :])\n",
    "                gettx9layer = Lambda(lambda x: x[:, 9, :])\n",
    "                gettx10layer = Lambda(lambda x: x[:, 10, :])\n",
    "                gettx11layer = Lambda(lambda x: x[:, 11, :])\n",
    "                gettx12layer = Lambda(lambda x: x[:, 12, :])\n",
    "                gettx13layer = Lambda(lambda x: x[:, 13, :])\n",
    "                gettx14layer = Lambda(lambda x: x[:, 14, :])\n",
    "                gettx15layer = Lambda(lambda x: x[:, 15, :])\n",
    "                gettx16layer = Lambda(lambda x: x[:, 16, :])\n",
    "                gettx17layer = Lambda(lambda x: x[:, 17, :])\n",
    "                gettx18layer = Lambda(lambda x: x[:, 18, :])\n",
    "                gettx19layer = Lambda(lambda x: x[:, 19, :])\n",
    "                gettx20layer = Lambda(lambda x: x[:, 20, :])\n",
    "                gettx21layer = Lambda(lambda x: x[:, 21, :])\n",
    "                gettx22layer = Lambda(lambda x: x[:, 22, :])\n",
    "                gettx23layer = Lambda(lambda x: x[:, 23, :])\n",
    "                gettx24layer = Lambda(lambda x: x[:, 24, :])\n",
    "                gettx25layer = Lambda(lambda x: x[:, 25, :])\n",
    "                gettx26layer = Lambda(lambda x: x[:, 26, :])\n",
    "                gettx27layer = Lambda(lambda x: x[:, 27, :])\n",
    "                gettx28layer = Lambda(lambda x: x[:, 28, :])\n",
    "                gettx29layer = Lambda(lambda x: x[:, 29, :])\n",
    "                gettx30layer = Lambda(lambda x: x[:, 30, :])\n",
    "                gettx31layer = Lambda(lambda x: x[:, 31, :])\n",
    "                if i == 1:\n",
    "                    outputtxsplit = gettx1layer(image_dropout)\n",
    "                elif i == 2:\n",
    "                    outputtxsplit = gettx2layer(image_dropout)\n",
    "                elif i==3:\n",
    "                    outputtxsplit = gettx3layer(image_dropout)\n",
    "                elif i == 4:\n",
    "                    outputtxsplit = gettx4layer(image_dropout)\n",
    "                elif i==5:\n",
    "                    outputtxsplit = gettx5layer(image_dropout)\n",
    "                elif i == 6:\n",
    "                    outputtxsplit = gettx6layer(image_dropout)\n",
    "                elif i==7:\n",
    "                    outputtxsplit = gettx7layer(image_dropout)\n",
    "                elif i == 8:\n",
    "                    outputtxsplit = gettx8layer(image_dropout)\n",
    "                elif i==9:\n",
    "                    outputtxsplit = gettx9layer(image_dropout)\n",
    "                elif i == 10:\n",
    "                    outputtxsplit = gettx10layer(image_dropout)\n",
    "                elif i==11:\n",
    "                    outputtxsplit = gettx11layer(image_dropout)\n",
    "                elif i == 12:\n",
    "                    outputtxsplit = gettx12layer(image_dropout)\n",
    "                elif i==13:\n",
    "                    outputtxsplit = gettx13layer(image_dropout)\n",
    "                elif i == 14:\n",
    "                    outputtxsplit = gettx14layer(image_dropout)\n",
    "                elif i==15:\n",
    "                    outputtxsplit = gettx15layer(image_dropout)\n",
    "                elif i == 16:\n",
    "                    outputtxsplit = gettx16layer(image_dropout)\n",
    "                elif i==17:\n",
    "                    outputtxsplit = gettx17layer(image_dropout)\n",
    "                elif i == 18:\n",
    "                    outputtxsplit = gettx18layer(image_dropout)\n",
    "                elif i==19:\n",
    "                    outputtxsplit = gettx19layer(image_dropout)\n",
    "                elif i == 20:\n",
    "                    outputtxsplit = gettx20layer(image_dropout)\n",
    "                elif i==21:\n",
    "                    outputtxsplit = gettx21layer(image_dropout)\n",
    "                elif i == 22:\n",
    "                    outputtxsplit = gettx22layer(image_dropout)\n",
    "                elif i==23:\n",
    "                    outputtxsplit = gettx23layer(image_dropout)\n",
    "                elif i == 24:\n",
    "                    outputtxsplit = gettx24layer(image_dropout)\n",
    "                elif i==25:\n",
    "                    outputtxsplit = gettx25layer(image_dropout)\n",
    "                elif i == 26:\n",
    "                    outputtxsplit = gettx26layer(image_dropout)\n",
    "                elif i==27:\n",
    "                    outputtxsplit = gettx27layer(image_dropout)\n",
    "                elif i == 28:\n",
    "                    outputtxsplit = gettx28layer(image_dropout)\n",
    "                elif i==29:\n",
    "                    outputtxsplit = gettx29layer(image_dropout)\n",
    "                elif i == 30:\n",
    "                    outputtxsplit = gettx30layer(image_dropout)\n",
    "                else:\n",
    "                    outputtxsplit = gettx31layer(image_dropout)\n",
    "                shape_im = Permute((2, 1))(outputtxsplit)\n",
    "                dim_change_im = Dense(units=1,\n",
    "                                    kernel_regularizer=l2(regularizer))(shape_im)\n",
    "                pre_merge_txt = Permute((2, 1))(dim_change_im)\n",
    "                #pre_merge_txt = Reshape((1, embedding_size))(outputtxsplit)\n",
    "                recurrent_inputs = [pre_merge_txt, pre_merge]\n",
    "                merged_input = Add()(recurrent_inputs)\n",
    "                #merged_input = Reshape((1, embedding_size), name='reshape3')(merged_input_temp)\n",
    "            lstm_out = LSTM(units=hidden_size,#[:, i, :]\n",
    "                            recurrent_regularizer=l2(regularizer),\n",
    "                            kernel_regularizer=l2(regularizer),\n",
    "                            bias_regularizer=l2(regularizer),\n",
    "                            return_sequences=True,\n",
    "                            name='recurrent_network' + str(i))(merged_input)\n",
    "            if i == 0:\n",
    "                lstm_out_final = Concatenate(axis=1)([lstm_out, lstm_out])\n",
    "            else:\n",
    "                lstm_out_final = Concatenate(axis=1)([lstm_out_final, lstm_out])\n",
    "    else:\n",
    "        raise Exception('Invalid rnn name')\n",
    "    getoutbelayer = Lambda(lambda x: x[:, 1:, :])\n",
    "    output_be = Reshape((max_token_length+1, embedding_size))(lstm_out_final)\n",
    "    output_bee = getoutbelayer(output_be)\n",
    "    output = TimeDistributed(Dense(units=vocabulary_size,\n",
    "                                    kernel_regularizer=l2(regularizer),\n",
    "                                    activation='softmax'),\n",
    "                                    name='output')(output_bee)\n",
    "\n",
    "    inputs = [text_input, image_input]\n",
    "    model = Model(inputs=inputs, outputs=output)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = NIC(16, 1024)\n",
    "plot_model(model, '../images/NIC.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.applications.vgg16 import VGG16\n",
    "\n",
    "model = VGG16(weights='imagenet', include_top=True, input_shape = (224, 224, 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_model(model, to_file='model.png', show_shapes=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k, v in train_descriptions.items():\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.preprocessing.text import Tokenizer\n",
    "\n",
    "    t = Tokenizer()  # all without .\n",
    "    text = \"<start> Tomorrow will be cold. <end>\"\n",
    "    text = text.replace(\".\", \" .\")\n",
    "    t.fit_on_texts([text])\n",
    "    print(t.word_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract features from each photo in the directory\n",
    "def extract_features(filename):\n",
    "    # load the model\n",
    "    model = VGG16()\n",
    "    # re-structure the model\n",
    "    model.layers.pop()\n",
    "    model = Model(inputs=model.inputs, outputs=model.layers[-1].output)\n",
    "    # load the photo\n",
    "    image = load_img(filename, target_size=(224, 224))\n",
    "    # convert the image pixels to a numpy array\n",
    "    image = img_to_array(image)\n",
    "    # reshape data for the model\n",
    "    image = image.reshape((1, image.shape[0], image.shape[1], image.shape[2]))\n",
    "    # prepare the image for the VGG model\n",
    "    image = preprocess_input(image)\n",
    "    # get features\n",
    "    feature = model.predict(image, verbose=0)\n",
    "    return feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Embedding, Input\n",
    "from keras.layers import BatchNormalization, Dense, RepeatVector\n",
    "from keras.applications.vgg16 import VGG16\n",
    "\n",
    "# The top layer is the last layer\n",
    "image_model = VGG16(weights='imagenet')\n",
    "# Fix the weights\n",
    "for layer in image_model.layers:\n",
    "    layer.trainable = False\n",
    "\n",
    "embedding_size = 300\n",
    "dense_input = BatchNormalization(axis=-1)(image_model.output)\n",
    "image_dense = Dense(units=embedding_size)(dense_input)  # FC layer\n",
    "# Add a timestep dimension to match LSTM's input size\n",
    "image_embedding = RepeatVector(1)(image_dense)\n",
    "image_input = image_model.input\n",
    "\n",
    "vocab_size = 2536\n",
    "embedding_size = 300\n",
    "\n",
    "sentence_input = Input(shape=[34])\n",
    "word_embedding = Embedding(\n",
    "    input_dim=vocab_size, output_dim=embedding_size)(sentence_input)\n",
    "\n",
    "from keras.layers import (BatchNormalization, Concatenate, Dense, LSTM,\n",
    "                          TimeDistributed)\n",
    "from keras.models import Model\n",
    "from keras.optimizers import Adam\n",
    "\n",
    "sequence_input = Concatenate(axis=1)([image_embedding, word_embedding])\n",
    "\n",
    "learning_rate = 0.00051\n",
    "lstm_output_size = 300\n",
    "vocab_size = 2536\n",
    "lstm_layers = 3\n",
    "dropout_rate = 0.22\n",
    "input_ = sequence_input\n",
    "\n",
    "for _ in range(lstm_layers):\n",
    "    input_ = BatchNormalization(axis=-1)(input_)\n",
    "    lstm_out = LSTM(\n",
    "        units=lstm_output_size,\n",
    "        return_sequences=True,\n",
    "        dropout=dropout_rate,\n",
    "        recurrent_dropout=dropout_rate)(input_)\n",
    "    input_ = lstm_out\n",
    "sequence_output = TimeDistributed(Dense(units=vocab_size))(lstm_out)\n",
    "\n",
    "model = Model(inputs=[image_input, sentence_input], outputs=sequence_output)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SVG(model_to_dot(model, show_shapes=True, show_layer_names=True).create(prog='dot', format='svg'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "usage: ipykernel_launcher.py [-h] [-i IMAGE] [-m MODEL]\n",
      "ipykernel_launcher.py: error: unrecognized arguments: -f /Users/Divyansh/Library/Jupyter/runtime/kernel-421658b4-8964-4bee-84bf-e58e86f851f8.json\n"
     ]
    },
    {
     "ename": "SystemExit",
     "evalue": "2",
     "output_type": "error",
     "traceback": [
      "An exception has occurred, use %tb to see the full traceback.\n",
      "\u001b[0;31mSystemExit\u001b[0m\u001b[0;31m:\u001b[0m 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/Divyansh/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py:2870: UserWarning: To exit: use 'exit', 'quit', or Ctrl-D.\n",
      "  warn(\"To exit: use 'exit', 'quit', or Ctrl-D.\", stacklevel=1)\n"
     ]
    }
   ],
   "source": [
    "from pickle import load\n",
    "import numpy as np\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.applications.vgg16 import VGG16\n",
    "from keras.preprocessing.image import load_img\n",
    "from keras.preprocessing.image import img_to_array\n",
    "from keras.applications.vgg16 import preprocess_input\n",
    "from keras.models import Model\n",
    "from keras.models import load_model\n",
    "from nltk.translate.bleu_score import corpus_bleu\n",
    "\n",
    "from CapGenerator import load_data as ld\n",
    "from CapGenerator import generate_model as gen\n",
    "import argparse\n",
    "\n",
    "# extract features from each photo in the directory\n",
    "def extract_features(filename):\n",
    "  # load the model\n",
    "  model = VGG16()\n",
    "  # re-structure the model\n",
    "  model.layers.pop()\n",
    "  model = Model(inputs=model.inputs, outputs=model.layers[-1].output)\n",
    "  # load the photo\n",
    "  image = load_img(filename, target_size=(224, 224))\n",
    "  # convert the image pixels to a numpy array\n",
    "  image = img_to_array(image)\n",
    "  # reshape data for the model\n",
    "  image = image.reshape((1, image.shape[0], image.shape[1], image.shape[2]))\n",
    "  # prepare the image for the VGG model\n",
    "  image = preprocess_input(image)\n",
    "  # get features\n",
    "  feature = model.predict(image, verbose=0)\n",
    "  return feature\n",
    "\n",
    "# generate a description for an image\n",
    "def generate_desc(model, tokenizer, photo, index_word, max_length, beam_size=5):\n",
    "\n",
    "  captions = [['startseq', 0.0]]\n",
    "  # seed the generation process\n",
    "  in_text = 'startseq'\n",
    "  # iterate over the whole length of the sequence\n",
    "  for i in range(max_length):\n",
    "    all_caps = []\n",
    "    # expand each current candidate\n",
    "    for cap in captions:\n",
    "      sentence, score = cap\n",
    "      # if final word is 'end' token, just add the current caption\n",
    "      if sentence.split()[-1] == 'endseq':\n",
    "        all_caps.append(cap)\n",
    "        continue\n",
    "      # integer encode input sequence\n",
    "      sequence = tokenizer.texts_to_sequences([sentence])[0]\n",
    "      # pad input\n",
    "      sequence = pad_sequences([sequence], maxlen=max_length)\n",
    "      # predict next words\n",
    "      y_pred = model.predict([photo,sequence], verbose=0)[0]\n",
    "      # convert probability to integer\n",
    "      yhats = np.argsort(y_pred)[-beam_size:]\n",
    "\n",
    "      for j in yhats:\n",
    "        # map integer to word\n",
    "        word = index_word.get(j)\n",
    "        # stop if we cannot map the word\n",
    "        if word is None:\n",
    "          continue\n",
    "        # Add word to caption, and generate log prob\n",
    "        caption = [sentence + ' ' + word, score + np.log(y_pred[j])]\n",
    "        all_caps.append(caption)\n",
    "\n",
    "    # order all candidates by score\n",
    "    ordered = sorted(all_caps, key=lambda tup:tup[1], reverse=True)\n",
    "    captions = ordered[:beam_size]\n",
    "\n",
    "  return captions\n",
    "\n",
    "# evaluate the skill of the model\n",
    "def evaluate_model(model, descriptions, photos, tokenizer, index_word, max_length):\n",
    "  actual, predicted = list(), list()\n",
    "  # step over the whole set\n",
    "  for key, desc_list in descriptions.items():\n",
    "    # generate description\n",
    "    yhat = generate_desc(model, tokenizer, photos[key], index_word, max_length)[0]\n",
    "    # store actual and predicted\n",
    "    references = [d.split() for d in desc_list]\n",
    "    actual.append(references)\n",
    "    # Use best caption\n",
    "    predicted.append(yhat[0].split())\n",
    "  # calculate BLEU score\n",
    "  print('BLEU-1: %f' % corpus_bleu(actual, predicted, weights=(1.0, 0, 0, 0)))\n",
    "  print('BLEU-2: %f' % corpus_bleu(actual, predicted, weights=(0.5, 0.5, 0, 0)))\n",
    "  print('BLEU-3: %f' % corpus_bleu(actual, predicted, weights=(0.3, 0.3, 0.3, 0)))\n",
    "  print('BLEU-4: %f' % corpus_bleu(actual, predicted, weights=(0.25, 0.25, 0.25, 0.25)))\n",
    "\n",
    "def eval_test_set(model, descriptions, photos, tokenizer, index_word, max_length):\n",
    "  actual, predicted = list(), list()\n",
    "  # step over the whole set\n",
    "  for key, desc_list in descriptions.items():\n",
    "    # generate description\n",
    "    yhat = generate_desc(model, tokenizer, photos[key], index_word, max_length)[0]\n",
    "    # store actual and predicted\n",
    "    references = [d.split() for d in desc_list]\n",
    "    actual.append(references)\n",
    "    # Use best caption\n",
    "    predicted.append(yhat[0].split())\n",
    "  predicted = sorted(predicted)\n",
    "  actual = [x for _,x in sorted(zip(actual,predicted))]\n",
    "\n",
    "if __name__ == '__main__':\n",
    "\n",
    "  parser = argparse.ArgumentParser(description='Generate image captions')\n",
    "  parser.add_argument(\"-i\", \"--image\", help=\"Input image path\")\n",
    "  parser.add_argument(\"-m\", \"--model\", help=\"model checkpoint\")\n",
    "  args = parser.parse_args()\n",
    "\n",
    "\n",
    "  # load the tokenizer\n",
    "  tokenizer = load(open('models/tokenizer.pkl', 'rb'))\n",
    "  index_word = load(open('models/index_word.pkl', 'rb'))\n",
    "  # pre-define the max sequence length (from training)\n",
    "  max_length = 34\n",
    "\n",
    "  # load the model\n",
    "  if args.model:\n",
    "    filename = args.model\n",
    "  else:\n",
    "    filename = 'models/model-ep005-loss3.504-val_loss3.893.h5'\n",
    "  model = load_model(filename)\n",
    "\n",
    "  if args.image:\n",
    "    # load and prepare the photograph\n",
    "    photo = extract_features(args.image)\n",
    "    # generate description\n",
    "    captions = generate_desc(model, tokenizer, photo, index_word, max_length)\n",
    "    for cap in captions:\n",
    "      # remove start and end tokens\n",
    "      seq = cap[0].split()[1:-1]\n",
    "      desc = ' '.join(seq)\n",
    "      print('{} [log prob: {:1.2f}]'.format(desc,cap[1]))\n",
    "  else:\n",
    "    # load test set\n",
    "    test_features, test_descriptions = ld.prepare_dataset('test')[1]\n",
    "\n",
    "    # evaluate model\n",
    "    evaluate_model(model, test_descriptions, test_features, tokenizer, index_word, max_length)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Anaconda",
   "language": "python",
   "name": "anaconda"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
